import re # 正则表达式
from bs4 import BeautifulSoup # 网页解析，获取数据
import urllib.request, urllib.error # 判定URL，获取数据
# import xlwt # 进行Excel操作

findLink = re.compile(r'<a href="(.*?)">')   # 创建正则表达式对象，表示规则
findImgSrc = re.compile(r'<img.*src="(.*?)"', re.S)  # 图片路径，re.S 忽略换行符
findTitle = re.compile(r'<span class="title">(.*?)</span>')
findRating = re.compile(r'<span class="rating_num" property="v:average">(.*)</span>')
findJudge = re.compile(r'<span>(\d*)人评价</span>')
findInq = re.compile(r'<span class="inq">(.*)</span>')
findBd = re.compile(r'<p class="">(.*)</p>', re.S)


# python 左必右开区间
def main():
  baseurl = 'https://movie.douban.com/top250?start='
  # 1. 获取网页
  savepath = ".\\douban.xls"
  getData(baseurl)
  # 2. 解析数据

def getData(baseurl):
  datalist = []
  for i in range(0, 1):
    url = baseurl + str(i * 25)
    html = askUrl(url)
    # 解析
    soup = BeautifulSoup(html, "html.parser")
    for item in soup.find_all('div', class_="item"): #查找符合要求的字符串形成列表
      #print(item)
      data = []
      item = str(item)

      link = re.findall(findLink, item)[0]
      data.append(link)
      imgSrc = re.findall(findImgSrc, item)[0]
      data.append(imgSrc)
      titles = re.findall(findTitle, item)
      data.append(titles[0])
      if (len(titles) == 2):
        otitle = titles[1].replace('/', "") # 去掉无关符号
        data.append(otitle)
      else:
        data.append(" ")  # 外国名字留空
      rating = re.findall(findRating, item)[0]
      data.append(rating)
      judge = re.findall(findJudge, item)[0]
      data.append((judge))
      inq = re.findall(findInq, item)
      if len(inq) != 0:
        inq = inq[0].replace("。", "")
      else:
        data.append(" ")
      data.append(inq)
      bd = re.findall(findBd, item)[0]
      bd = re.sub('<br(\s+)?/>(\s+)?', " ", bd)
      bd = re.sub('/', " ", bd)
      data.append(bd.strip())

      datalist.append(data)  # 把处理好的一部电影插入datalist中
  print(datalist)
  # return datalist

def askUrl(url):
  headers = {
    "User-Agent": "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/89.0.4389.114 Safari/537.36 Edg/89.0.774.68"
  }
  request = urllib.request.Request(url, headers=headers)
  html = ""
  try:
    response = urllib.request.urlopen(request)
    html = response.read().decode("utf-8")
    #print(html)
  except urllib.error.URLError as e:
    if hasattr(e, "code"):
      print(e.code)
    if hasattr(e, "reason"):
      print(e.reason)
  return html


if __name__ == "__main__":
  main()